﻿using CsQuery;
using log4net;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading.Tasks;

namespace NugetDownloader.Spider
{
    class Spider
    {
        ILog log = LogManager.GetLogger(typeof(Spider));

        public async Task<List<string>> GetPackagesUrl(string query, int page, bool prerel = false)
        {
            List<string> results = new List<string>();
            var url = $"https://www.nuget.org/packages?page={page}" + (string.IsNullOrEmpty(query) ? "": $"&q={query}") + (prerel ? "" : $"&prerel=false");
            using(var wc = new WebClient())
            {
                var html = wc.DownloadString(url);
                if (!string.IsNullOrEmpty(html))
                {
                    var dom = CQ.Create(html);
                    var articles = dom.Select("article.package a.package-title");
                    articles.Each((i, e) => {
                        log.Info($"GetPackagesUrl:{e.GetAttribute("href")}");
                        results.Add(e.GetAttribute("href"));
                    });
                }
            }
            return results;
        }

        public async Task<List<string>> GetPackageDownloadUrls(string packageUrl)
        {
            List<string> results = new List<string>();
            using (var wc = new WebClient())
            {
                var html = wc.DownloadString(packageUrl);
                if (!string.IsNullOrEmpty(html))
                {
                    var dom = CQ.Create(html);
                    var articles = dom.Select("a[data-track=outbound-manual-download]");
                    articles.Each((i, e) => {
                        log.Info($"GetPackageDownloadUrls:{e.GetAttribute("href")}");
                        results.Add(e.GetAttribute("href"));
                        //Console.WriteLine("GetPackageDownloadUrl: {0}", e.GetAttribute("href"));
                    });
                }
            }
            return results;
        }

        public async Task<List<string>> GetPackageDownloadUrls(string query, int page, bool prerel = false)
        {
            var packagesUrl = await GetPackagesUrl(query, page);
            log.Info($"GetPackageDownloadUrls:page={page}, count = {packagesUrl.Count}");
            List<string> results = new List<string>();
            foreach (var item in packagesUrl)
            {
                var url = await GetPackageDownloadUrls($"https://www.nuget.org{item}");
                results.AddRange(url);
            }

            return results;
        }
    }
}
